useNavigate isn't a Higher Order Component, it's a React hook, it must be used within a function component or custom React hook.

Move it into the function body and fix the navigation.

const CartDropDown = ({ cartItems }) => {
  const navigate = useNavigate(); // <-- use hook in component

  return (
    <div className="cart-dropdown">
      <div className="cart-items">
        {cartItems.length
          ? cartItems.map(cartItem => (
              <CartItem key={cartItem.id} item={cartItem} />
            ))
          : <span className="empty-message">Your cart is empty</span>
        }
        <CustomButton
          onClick{() => navigate('/checkout')} // <-- call navigate in handler
        >
          GO TO CHECKOUT
        </CustomButton>
      </div>
    </div>
  )
}

const mapStateToProps = createStructuredSelector({
  cartItems: selectCartItems
})

export default connect(mapStateToProps)(CartDropDown);

A react hook must be inside the component. You get a navigate function from useNavigate(), and then you call that to navigate.

import { useNavigate } from 'react-router-dom'
import React from 'react'
import CustomButton from '../custom-button/custom-button.component'
import './cart-dropdown.styles.scss'
import CartItem from '../cart-item/cart-item.component'
import { connect } from 'react-redux'
import { selectCartItems } from '../../redux/cart/cart.selectors'
import { createStructuredSelector } from 'reselect'

const CartDropDown = ({ cartItems }) => {
  const navigate = useNavigate() // <-- hooks must be INSIDE the component

  const onCustomButtonClick = (event) => {
    //window.history.push('/checkout') // <-- no
    navigate('/checkout')
  }

  return (
    <div className="cart-dropdown">
      <div className="cart-items">
        {
          cartItems.length
            ? cartItems.map(cartItem => <CartItem key={cartItem.id} item={cartItem} />)
            : <span className="empty-message" > Your cart is empty</span>
        }
        <CustomButton onClick={onCustomButtonClick}> GO TO CHECKOUT </CustomButton>
      </div>
    </div>)
}

const mapStateToProps = createStructuredSelector({
  cartItems: selectCartItems
})

//export default useNavigate(connect(mapStateToProps)(CartDropDown))
export default CartDropDown